{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.数据变换\n",
    "## 4.1 对于标称型属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0       208500\n",
      "1       181500\n",
      "2       223500\n",
      "3       140000\n",
      "4       250000\n",
      "5       143000\n",
      "6       307000\n",
      "7       200000\n",
      "8       129900\n",
      "9       118000\n",
      "10      129500\n",
      "11      345000\n",
      "12      144000\n",
      "13      279500\n",
      "14      157000\n",
      "15      132000\n",
      "16      149000\n",
      "17       90000\n",
      "18      159000\n",
      "19      139000\n",
      "20      325300\n",
      "21      139400\n",
      "22      230000\n",
      "23      129900\n",
      "24      154000\n",
      "25      256300\n",
      "26      134800\n",
      "27      306000\n",
      "28      207500\n",
      "29       68500\n",
      "         ...  \n",
      "1428    192140\n",
      "1429    143750\n",
      "1430     64500\n",
      "1431    186500\n",
      "1432    160000\n",
      "1433    174000\n",
      "1434    120500\n",
      "1435    394617\n",
      "1436    149700\n",
      "1437    197000\n",
      "1438    191000\n",
      "1439    149300\n",
      "1440    310000\n",
      "1441    121000\n",
      "1442    179600\n",
      "1443    129000\n",
      "1444    157900\n",
      "1445    240000\n",
      "1446    112000\n",
      "1447     92000\n",
      "1448    136000\n",
      "1449    287090\n",
      "1450    145000\n",
      "1451     84500\n",
      "1452    185000\n",
      "1453    175000\n",
      "1454    210000\n",
      "1455    266500\n",
      "1456    142125\n",
      "1457    147500\n",
      "Name: SalePrice, Length: 1458, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>...</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0          60       RL         65.0     8450   Pave  None      Reg   \n",
       "1          20       RL         80.0     9600   Pave  None      Reg   \n",
       "2          60       RL         68.0    11250   Pave  None      IR1   \n",
       "3          70       RL         60.0     9550   Pave  None      IR1   \n",
       "4          60       RL         84.0    14260   Pave  None      IR1   \n",
       "\n",
       "  LandContour Utilities LotConfig      ...       ScreenPorch PoolArea PoolQC  \\\n",
       "0         Lvl    AllPub    Inside      ...                 0        0   None   \n",
       "1         Lvl    AllPub       FR2      ...                 0        0   None   \n",
       "2         Lvl    AllPub    Inside      ...                 0        0   None   \n",
       "3         Lvl    AllPub    Corner      ...                 0        0   None   \n",
       "4         Lvl    AllPub       FR2      ...                 0        0   None   \n",
       "\n",
       "  Fence MiscFeature MiscVal  MoSold  YrSold  SaleType  SaleCondition  \n",
       "0  None        None       0       2    2008        WD         Normal  \n",
       "1  None        None       0       5    2007        WD         Normal  \n",
       "2  None        None       0       9    2008        WD         Normal  \n",
       "3  None        None       0       2    2006        WD        Abnorml  \n",
       "4  None        None       0      12    2008        WD         Normal  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"./clean_data/train_afterclean.csv\")\n",
    "print(train.SalePrice)\n",
    "test = pd.read_csv(\"./clean_data/test_afterclean.csv\")\n",
    "alldata = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'], test.loc[:,'MSSubClass':'SaleCondition']), ignore_index=True)\n",
    "alldata.shape\n",
    "alldata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理序列型标称数据\n",
    "ordinalList = ['ExterQual', 'ExterCond', 'GarageQual', 'GarageCond','PoolQC',\\\n",
    "              'FireplaceQu', 'KitchenQual', 'HeatingQC', 'BsmtQual','BsmtCond']\n",
    "ordinalmap = {'Ex': 5,'Gd': 4,'TA': 3,'Fa': 2,'Po': 1,'None': 0}\n",
    "for c in ordinalList:\n",
    "    alldata[c] = alldata[c].map(ordinalmap) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 其他类别属性——标签化\n",
    "alldata['BsmtExposure'] = alldata['BsmtExposure'].map({'None':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4})    \n",
    "alldata['BsmtFinType1'] = alldata['BsmtFinType1'].map({'None':0, 'Unf':1, 'LwQ':2,'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})\n",
    "alldata['BsmtFinType2'] = alldata['BsmtFinType2'].map({'None':0, 'Unf':1, 'LwQ':2,'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})\n",
    "alldata['Functional'] = alldata['Functional'].map({'Maj2':1, 'Sev':2, 'Min2':3, 'Min1':4, 'Maj1':5, 'Mod':6, 'Typ':7})\n",
    "alldata['GarageFinish'] = alldata['GarageFinish'].map({'None':0, 'Unf':1, 'RFn':2, 'Fin':3})\n",
    "alldata['Fence'] = alldata['Fence'].map({'MnWw':0, 'GdWo':1, 'MnPrv':2, 'GdPrv':3, 'None':4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 部分属性创建二值新属性\n",
    "MasVnrType_Any = alldata.MasVnrType.replace({'BrkCmn': 1,'BrkFace': 1,'CBlock': 1,'Stone': 1,'None': 0})\n",
    "MasVnrType_Any.name = 'MasVnrType_Any' #修改该series的列名\n",
    "SaleCondition_PriceDown = alldata.SaleCondition.replace({'Abnorml': 1,'Alloca': 1,'AdjLand': 1,'Family': 1,'Normal': 0,'Partial': 0})\n",
    "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown' #修改该series的列名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "alldata = alldata.replace({'CentralAir': {'Y': 1,'N': 0}})\n",
    "alldata = alldata.replace({'PavedDrive': {'Y': 1,'P': 0,'N': 0}})\n",
    "newer_dwelling = alldata['MSSubClass'].map({20: 1,30: 0,40: 0,45: 0,50: 0,60: 1,70: 0,75: 0,80: 0,85: 0,90: 0,120: 1,150: 0,160: 0,180: 0,190: 0})\n",
    "newer_dwelling.name= 'newer_dwelling' #修改该series的列名\n",
    "alldata['MSSubClass'] = alldata['MSSubClass'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Neighborhood_Good = pd.DataFrame(np.zeros((alldata.shape[0],1)), columns=['Neighborhood_Good'])\n",
    "Neighborhood_Good[alldata.Neighborhood=='NridgHt'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='Crawfor'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='StoneBr'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='Somerst'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='NoRidge'] = 1\n",
    "# Neighborhood_Good = (alldata['Neighborhood'].isin(['StoneBr','NoRidge','NridgHt','Timber','Somerst']))*1 #(效果没有上面好)\n",
    "Neighborhood_Good.name='Neighborhood_Good'# 将该变量加入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "season = (alldata['MoSold'].isin([5,6,7]))*1 #(@@@@@)\n",
    "season.name='season'\n",
    "alldata['MoSold'] = alldata['MoSold'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对与“质量——Qual”“条件——Cond”属性，构造新属性\n",
    "# 处理OverallQual：将该属性分成两个子属性，以5为分界线，大于5及小于5的再分别以序列\n",
    "overall_poor_qu = alldata.OverallQual.copy()# Series类型\n",
    "overall_poor_qu = 5 - overall_poor_qu\n",
    "overall_poor_qu[overall_poor_qu<0] = 0\n",
    "overall_poor_qu.name = 'overall_poor_qu'\n",
    "overall_good_qu = alldata.OverallQual.copy()\n",
    "overall_good_qu = overall_good_qu - 5\n",
    "overall_good_qu[overall_good_qu<0] = 0\n",
    "overall_good_qu.name = 'overall_good_qu'\n",
    " \n",
    "# 处理OverallCond ：将该属性分成两个子属性，以5为分界线，大于5及小于5的再分别以序列\n",
    "overall_poor_cond = alldata.OverallCond.copy()# Series类型\n",
    "overall_poor_cond = 5 - overall_poor_cond\n",
    "overall_poor_cond[overall_poor_cond<0] = 0\n",
    "overall_poor_cond.name = 'overall_poor_cond'\n",
    "overall_good_cond = alldata.OverallCond.copy()\n",
    "overall_good_cond = overall_good_cond - 5\n",
    "overall_good_cond[overall_good_cond<0] = 0\n",
    "overall_good_cond.name = 'overall_good_cond'\n",
    " \n",
    "# 处理ExterQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "exter_poor_qu = alldata.ExterQual.copy()\n",
    "exter_poor_qu[exter_poor_qu<3] = 1\n",
    "exter_poor_qu[exter_poor_qu>=3] = 0\n",
    "exter_poor_qu.name = 'exter_poor_qu'\n",
    "exter_good_qu = alldata.ExterQual.copy()\n",
    "exter_good_qu[exter_good_qu<=3] = 0\n",
    "exter_good_qu[exter_good_qu>3] = 1\n",
    "exter_good_qu.name = 'exter_good_qu'\n",
    " \n",
    "# 处理ExterCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "exter_poor_cond = alldata.ExterCond.copy()\n",
    "exter_poor_cond[exter_poor_cond<3] = 1\n",
    "exter_poor_cond[exter_poor_cond>=3] = 0\n",
    "exter_poor_cond.name = 'exter_poor_cond'\n",
    "exter_good_cond = alldata.ExterCond.copy()\n",
    "exter_good_cond[exter_good_cond<=3] = 0\n",
    "exter_good_cond[exter_good_cond>3] = 1\n",
    "exter_good_cond.name = 'exter_good_cond'\n",
    " \n",
    "# 处理BsmtCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "bsmt_poor_cond = alldata.BsmtCond.copy()\n",
    "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n",
    "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n",
    "bsmt_poor_cond.name = 'bsmt_poor_cond'\n",
    "bsmt_good_cond = alldata.BsmtCond.copy()\n",
    "bsmt_good_cond[bsmt_good_cond<=3] = 0\n",
    "bsmt_good_cond[bsmt_good_cond>3] = 1\n",
    "bsmt_good_cond.name = 'bsmt_good_cond'\n",
    " \n",
    "# 处理GarageQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "garage_poor_qu = alldata.GarageQual.copy()\n",
    "garage_poor_qu[garage_poor_qu<3] = 1\n",
    "garage_poor_qu[garage_poor_qu>=3] = 0\n",
    "garage_poor_qu.name = 'garage_poor_qu'\n",
    "garage_good_qu = alldata.GarageQual.copy()\n",
    "garage_good_qu[garage_good_qu<=3] = 0\n",
    "garage_good_qu[garage_good_qu>3] = 1\n",
    "garage_good_qu.name = 'garage_good_qu'\n",
    " \n",
    "# 处理GarageCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "garage_poor_cond = alldata.GarageCond.copy()\n",
    "garage_poor_cond[garage_poor_cond<3] = 1\n",
    "garage_poor_cond[garage_poor_cond>=3] = 0\n",
    "garage_poor_cond.name = 'garage_poor_cond'\n",
    "garage_good_cond = alldata.GarageCond.copy()\n",
    "garage_good_cond[garage_good_cond<=3] = 0\n",
    "garage_good_cond[garage_good_cond>3] = 1\n",
    "garage_good_cond.name = 'garage_good_cond'\n",
    " \n",
    "# 处理KitchenQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "kitchen_poor_qu = alldata.KitchenQual.copy()\n",
    "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n",
    "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n",
    "kitchen_poor_qu.name = 'kitchen_poor_qu'\n",
    "kitchen_good_qu = alldata.KitchenQual.copy()\n",
    "kitchen_good_qu[kitchen_good_qu<=3] = 0\n",
    "kitchen_good_qu[kitchen_good_qu>3] = 1\n",
    "kitchen_good_qu.name = 'kitchen_good_qu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将构造的属性合并\n",
    "qu_list = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n",
    "                     exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n",
    "                     garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对与时间相关属性处理\n",
    "Xremoded = (alldata['YearBuilt']!=alldata['YearRemodAdd'])*1 #(@@@@@)\n",
    "Xrecentremoded = (alldata['YearRemodAdd']>=alldata['YrSold'])*1 #(@@@@@)\n",
    "XnewHouse = (alldata['YearBuilt']>=alldata['YrSold'])*1 #(@@@@@)\n",
    "XHouseAge = 2010 - alldata['YearBuilt']\n",
    "XTimeSinceSold = 2010 - alldata['YrSold']\n",
    "XYearSinceRemodel = alldata['YrSold'] - alldata['YearRemodAdd']\n",
    " \n",
    "Xremoded.name='Xremoded'\n",
    "Xrecentremoded.name='Xrecentremoded'\n",
    "XnewHouse.name='XnewHouse'\n",
    "XTimeSinceSold.name='XTimeSinceSold'\n",
    "XYearSinceRemodel.name='XYearSinceRemodel'\n",
    "XHouseAge.name='XHouseAge'\n",
    " \n",
    "year_list = pd.concat((Xremoded,Xrecentremoded,XnewHouse,XHouseAge,XTimeSinceSold,XYearSinceRemodel),axis=1)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "构造新属性'price_category'\n",
    "\n",
    "此处利用SVM支持向量机构造新属性\n",
    "\n",
    "class sklearn.svm.SVC(C=1.0, kernel=’rbf’, degree=3, gamma=’auto’, coef0=0.0, \n",
    "shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None,\n",
    "verbose=False, max_iter=-1, decision_function_shape=’ovr’, random_state=None)\n",
    "SVC参数解释 \n",
    "（1）C: 目标函数的惩罚系数C，用来平衡分类间隔margin和错分样本的，default C = 1.0； \n",
    "（2）kernel：参数选择有RBF, Linear, Poly, Sigmoid, 默认的是\"RBF\"; \n",
    "（3）degree：if you choose 'Poly' in param 2, this is effective, degree决定了多项式的最高次幂； \n",
    "（4）gamma：核函数的系数('Poly', 'RBF' and 'Sigmoid'), 默认是gamma = 1 / n_features; \n",
    "（5）coef0：核函数中的独立项，'RBF' and 'Poly'有效； \n",
    "（6）probablity: 可能性估计是否使用(true or false)； \n",
    "（7）shrinking：是否进行启发式； \n",
    "（8）tol（default = 1e - 3）: svm结束标准的精度; \n",
    "（9）cache_size: 制定训练所需要的内存（以MB为单位）； \n",
    "（10）class_weight: 每个类所占据的权重，不同的类设置不同的惩罚参数C, 缺省的话自适应； \n",
    "（11）verbose: 跟多线程有关，不大明白啥意思具体； \n",
    "（12）max_iter: 最大迭代次数，default = 1， if max_iter = -1, no limited; \n",
    "（13）decision_function_shape ： ‘ovo’ 一对一, ‘ovr’ 多对多  or None 无, default=None \n",
    "（14）random_state ：用于概率估计的数据重排时的伪随机数生成器的种子。 \n",
    " ps：7,8,9一般不考虑。 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "svm = SVC(C=100, gamma=0.0001, kernel='rbf')\n",
    " \n",
    "pc = pd.Series(np.zeros(train.shape[0]))\n",
    "pc[:] = 'pc1'\n",
    "pc[train.SalePrice >= 150000] = 'pc2'\n",
    "pc[train.SalePrice >= 220000] = 'pc3'\n",
    "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n",
    "X_t = pd.get_dummies(train.loc[:, columns_for_pc], sparse=True)\n",
    "svm.fit(X_t, pc)# 训练\n",
    "p = train.SalePrice/100000\n",
    " \n",
    "price_category = pd.DataFrame(np.zeros((alldata.shape[0],1)), columns=['pc'])\n",
    "X_t = pd.get_dummies(alldata.loc[:, columns_for_pc], sparse=True)\n",
    "pc_pred = svm.predict(X_t) # 预测\n",
    " \n",
    "price_category[pc_pred=='pc2'] = 1\n",
    "price_category[pc_pred=='pc3'] = 2\n",
    "price_category.name='price_category'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 连续数据离散化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n",
    "# 将年份对应映射\n",
    "# alldata.GarageYrBlt = alldata.GarageYrBlt.map(year_map) # 在数据填充时已经完成该转换了（因为必须先转化后再填充，否则会出错（可以想想到底为什么呢？））\n",
    "alldata.YearBuilt = alldata.YearBuilt.map(year_map)\n",
    "alldata.YearRemodAdd = alldata.YearRemodAdd.map(year_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 处理数值型属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简单函数 规范化 按照比例缩放\n",
    "numeric_feats = alldata.dtypes[alldata.dtypes != \"object\"].index\n",
    "t = alldata[numeric_feats].quantile(.75) # 取四分之三分位\n",
    "use_75_scater = t[t != 0].index\n",
    "alldata[use_75_scater] = alldata[use_75_scater]/alldata[use_75_scater].quantile(.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 标准化数据使符合正态分布\n",
    "from scipy.special import boxcox1p\n",
    " \n",
    "t = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',\n",
    "     '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF',\n",
    "     'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']\n",
    "# alldata.loc[:, t] = np.log1p(alldata.loc[:, t])\n",
    "train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"]) # 对于SalePrice 采用log1p较好---np.expm1(clf1.predict(X_test))\n",
    " \n",
    "lam = 0.15 # 100 * (1-lam)% confidence\n",
    "for feat in t:\n",
    "    alldata[feat] = boxcox1p(alldata[feat], lam)  # 对于其他属性，采用boxcox1p较好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将标称型变量二值化\n",
    "X = pd.get_dummies(alldata)\n",
    "X = X.fillna(X.mean())\n",
    " \n",
    "X = X.drop('Condition2_PosN', axis=1)\n",
    "X = X.drop('MSZoning_C (all)', axis=1)\n",
    "X = X.drop('MSSubClass_160', axis=1)\n",
    "X= pd.concat((X, newer_dwelling, season, year_list ,qu_list,MasVnrType_Any, \\\n",
    "              price_category,SaleCondition_PriceDown,Neighborhood_Good), axis=1)#SaleCondition_PriceDown,Neighborhood_Good已经加入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建新属性\n",
    "from itertools import product, chain\n",
    "# chain(iter1, iter2, ..., iterN):\n",
    "# 给出一组迭代器(iter1, iter2, ..., iterN)，此函数创建一个新迭代器来将所有的迭代器链接起来，\n",
    "# 返回的迭代器从iter1开始生成项，知道iter1被用完，然后从iter2生成项，这一过程会持续到iterN中所有的项都被用完。\n",
    "def poly(X):\n",
    "    areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']  # 5个\n",
    "    t = chain(qu_list.axes[1].get_values(),year_list.axes[1].get_values(),ordinalList,\n",
    "              ['MasVnrType_Any'])  #,'Neighborhood_Good','SaleCondition_PriceDown'\n",
    "    for a, t in product(areas, t):\n",
    "        x = X.loc[:, [a, t]].prod(1) # 返回各维数组的乘积\n",
    "        x.name = a + '_' + t\n",
    "        yield x\n",
    "# 带有 yield 的函数在 Python 中被称之为 generator（生成器）\n",
    "# 一个带有 yield 的函数就是一个 generator，它和普通函数不同，生成一个 generator 看起来像函数调用，\n",
    "# 但不会执行任何函数代码，直到对其调用 next()（在 for 循环中会自动调用 next()）才开始执行。\n",
    "# 虽然执行流程仍按函数的流程执行，但每执行到一个 yield 语句就会中断，并返回一个迭代值，\n",
    "# 下次执行时从 yield 的下一个语句继续执行。看起来就好像一个函数在正常执行的过程中被 yield 中断了数次，\n",
    "# 每次中断都会通过 yield 返回当前的迭代值。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1458, 460)\n"
     ]
    }
   ],
   "source": [
    "XP = pd.concat(poly(X), axis=1) # (2917, 155)\n",
    "X = pd.concat((X, XP), axis=1) # (2917, 466)\n",
    "X_train = X[:train.shape[0]]\n",
    "X_test = X[train.shape[0]:]\n",
    "print(X_train.shape)#(1458, 460)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "Y= train.SalePrice\n",
    "train_now = pd.concat([X_train,Y], axis=1)\n",
    "test_now = X_test\n",
    "# 将处理后的结果进行保存，用于接下来构建模型\n",
    "train_now.to_csv('./data_change/data_changetrain_afterchange.csv')\n",
    "test_now.to_csv('./data_change/test_afterchange.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
